/*
	Q房网在安居客指定楼盘占比
	feng 2015-08-04 星期二
*/

package main

import (
	"bufio"
	"bytes"
	"code.google.com/p/go.text/encoding/simplifiedchinese"
	"code.google.com/p/go.text/transform"
	"fmt"
	goq "github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	MainDomain = "http://beijing.anjuke.com"
	LOG        = "Anjuke"
	ErrLOG     = "Anjuke"
)

var (
	NumCPU    = runtime.NumCPU()
	waitgroup = &sync.WaitGroup{}
)

func init() {
	runtime.GOMAXPROCS(NumCPU)
}

func getCommunityList() (map[int]string, error) {
	f, e := os.Open("communitylist_config.txt")
	if e != nil {
		return nil, e
	} else {
		ret := make(map[int]string)
		rd := bufio.NewReader(f)
		for i := 0; ; i++ {
			line, _, e := rd.ReadLine()
			if e != nil {
				break
			}
			ret[i] = string(line[:])
		}
		return ret, nil
	}
}

//检查楼盘是否存在
func chkCommunityListValidate(CommunityList map[int]string) {
	for i := 0; i < len(CommunityList); i += NumCPU {
		for j := i; j < i+NumCPU && j < len(CommunityList); j++ {
			waitgroup.Add(1)
			communityname := CommunityList[j]
			go func(communityname string) {
				defer waitgroup.Done()
				kw := MainDomain + "/community/" + communityname + "W0QQrdZ1"
				doc, e := getDoc(kw)
				if e != nil {
					errlog(communityname + "抓取失败 错误代码:" + e.Error())
					return
				}
				foundOut := doc.Find("span:contains('找到北京小区') em").Text()
				if foundOut != "" && foundOut != "0" {
					outlog(communityname + "找到了")
				} else {
					errlog(communityname + "没找到")
				}
			}(communityname)
		}
		waitgroup.Wait()
		time.Sleep(time.Second * 1)
	}
	os.Exit(-1)
}

func main() {
	CommunityList, e := getCommunityList()
	if e != nil {
		s(e.Error())
		os.Exit(-1)
	}

	//chkCommunityListValidate(CommunityList)

	for i := 0; i < len(CommunityList); i += NumCPU {
		for j := i; j < i+NumCPU && j < len(CommunityList); j++ {
			waitgroup.Add(1)
			communityname := CommunityList[j]
			go func(communityname string) {
				defer waitgroup.Done()
				kw := MainDomain + "/community/" + communityname + "W0QQrdZ1"
				doc, e := getDoc(kw)
				if e != nil {
					errlog(communityname + "抓取失败 错误代码:" + e.Error())
					return
				}
				foundOut := doc.Find("span:contains('找到北京小区') em").Text()
				if foundOut != "" && foundOut != "0" {
					var (
						salenum   int //二手房源数
						Q_salenum int //Q房的人二手房源数
					)
					doc.Find(".list .list_item .details .t_b .t").EachWithBreak(func(i int, sec *goq.Selection) bool {
						tmp_communityname := sec.Text()
						if tmp_communityname == communityname {
							tmp_salenum := sec.Parent().Parent().Find("p:contains('二手房') a").Text()
							tmp_salenum = strings.Replace(tmp_salenum, "(", "", -1)
							tmp_salenum = strings.Replace(tmp_salenum, ")", "", -1)
							num_salenum, e := strconv.Atoi(tmp_salenum)
							if e == nil {
								salenum = num_salenum
							}
							return false
						} else {
							return true
						}
					})
					if salenum == 0 {
						return
					}
					Q_salenum = getCommunityInfoByAgent(communityname)

					outstr := fmt.Sprintf("%s,%d,%d,%.2f%%", U2G(communityname), salenum, Q_salenum, float64(Q_salenum)/float64(salenum)*100)
					outlog(outstr)
				} else {
					errlog(communityname + "没找到")
				}
			}(communityname)
		}
		waitgroup.Wait()
		time.Sleep(time.Second * 5)
	}
}

//通过楼盘获取经纪人数信息
func getCommunityInfoByAgent(communityname string) int {
	Q_salenum := 0
	url := MainDomain + "/tycoon/ppxxpp-st1?key=" + communityname	
	curpage := 1
	for {
		tmpurl := strings.Replace(url, "ppxxpp", fmt.Sprintf("p%d", curpage), 1)
		doc, e := getDoc(tmpurl)
		if e != nil {
			continue
		}
		//是否有内容
		if !doc.Find(".tycoon_list").HasClass("tycoon_list") {
			break
		}
		doc.Find(".tycoon_list ul li").Each(func(i int, sec *goq.Selection) {
			tmp_company := sec.Find("dd a:contains('Q房网')").Text()
			if tmp_company != "" { //是Q房网经纪人
				if agent_url, ok := sec.Find("dt a").Attr("href"); ok {
					Q_salenum += getSalenumInCommunity(agent_url, communityname)
				}
			}
		})
		//是否还有下一页
		if doc.Find(".multipage").Find(".current").Next().HasClass("nextpage") {
			break
		}
		curpage++
	}
	return Q_salenum
}

//获取经纪人在此楼盘的发房数
func getSalenumInCommunity(agent_url, communityname string) int {
	doc, e := getDoc(agent_url)
	if e != nil {
		return 0
	}
	Q_salenum := 0
	doc.Find("#ulCommList li a:contains('" + communityname + "')").EachWithBreak(func(i int, sec *goq.Selection) bool {
		tmp_communityname := sec.Text()
		if strings.Contains(tmp_communityname, communityname+"(") {
			tmp_Q_salenum := sec.Find("span").Text()
			tmp_Q_salenum = strings.Replace(tmp_Q_salenum, "(", "", -1)
			tmp_Q_salenum = strings.Replace(tmp_Q_salenum, ")", "", -1)
			num_Q_salenum, e := strconv.Atoi(tmp_Q_salenum)
			if e == nil {
				Q_salenum = num_Q_salenum
			}
			return false
		}
		return true
	})
	return Q_salenum
}

//获取doc
func getDoc(url string) (*goq.Document, error) {
	res, e := getResp(url)
	if e != nil {
		return nil, e
	}
	doc, e := goq.NewDocumentFromResponse(res)
	if e != nil {
		return nil, e
	}
	return doc, nil
}

//获取response
func getResp(url string) (*http.Response, error) {
	req, e := http.NewRequest("GET", url, nil)
	req.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	//req.Header.Add("Accept-Encoding", "gzip,deflate,sdch")
	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.8")
	req.Header.Add("Cache-Control", "max-age=0")
	req.Header.Add("Proxy-Connection", "keep-alive")
	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")
	c := &http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(25 * time.Second)
				c, err := net.DialTimeout(netw, addr, time.Second*20)
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	var resp *http.Response
	//尝试三次
	for retry := 0; retry < 3; retry++ {
		resp, e = c.Do(req)
		if e != nil {
			time.Sleep(time.Second * 1)
		} else {
			break
		}
	}
	if e != nil {
		return nil, e
	}
	return resp, nil
}

//gbk->utf8
func G2U(s string) string {
	ret, e := gbktoutf8([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

//utf8->gbk
func U2G(s string) string {
	ret, e := utf8togbk([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

func gbktoutf8(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}
func utf8togbk(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewEncoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}

func filterStr(str string) string {
	str = strings.Replace(str, "\r\n", "", -1)
	str = strings.Replace(str, "\r", "", -1)
	str = strings.Replace(str, "\n", "", -1)
	str = strings.Replace(str, "|", "", -1)
	str = strings.Replace(str, ",", "", -1)
	str = strings.TrimSpace(str)
	return str
}

func s(a ...interface{}) {
	fmt.Println(a)
}

//outlog
func outlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_log.csv", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}

//outlog
func errlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_err.txt", pwd, ErrLOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}
